﻿//////////////////////////////////////////////////////////////////
//
// Bruno\Pager.cs
//
// Copyright (c) 2012 Dan Pike.
//
// Permission is hereby granted, free of charge, to any person
// obtaining a copy of this software and associated documentation files
// (the "Software"), to deal in the Software without restriction,
// including without limitation the rights to use, copy, modify, merge,
// publish, distribute, sublicense, and/or sell copies of the Software,
// and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included
// in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
// OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
// IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
// CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
// TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
// SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
//
using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Security;
using System.Security.Policy;
using System.Security.Cryptography.X509Certificates;
using System.Security.Cryptography;
using System.Text;
using Utilities;
using System.Threading;
using System.Threading.Tasks;
using Ionic.Zlib;

namespace Bruno
{
   public class Pager
   {
      private static Logger logger_;
      private ASCIIEncoding encoding_;
      private string sessionId_;
      public string UserName;
      public string Password;
      public string LoginServer;
      private string codeName_;
      private DateTime codeDate_;

      private enum NodeLevel
      {
         None,
         LookingForHtml,
         LookingForBody,
         InBody,
         StripTail
      };

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      static Pager()
      {
         logger_ = Program.Logger;
         ServicePointManager.ServerCertificateValidationCallback =
            new RemoteCertificateValidationCallback(
               new RemoteCertificateValidationCallback(MyCertValidationCb));
         ServicePointManager.DefaultConnectionLimit = 16;
         ServicePointManager.Expect100Continue = false;
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      public Pager()
      {
         encoding_ = new ASCIIEncoding();
         codeName_ = string.Empty;
         codeDate_ = DateTime.MinValue;
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      /// <param name="sender"></param>
      /// <param name="certificate"></param>
      /// <param name="chain"></param>
      /// <param name="sslPolicyErrors"></param>
      /// <returns></returns>
      public static bool MyCertValidationCb(
            object sender, X509Certificate certificate,
            X509Chain chain, SslPolicyErrors sslPolicyErrors)
      {
         if ((sslPolicyErrors & SslPolicyErrors.RemoteCertificateChainErrors) ==
            SslPolicyErrors.RemoteCertificateChainErrors)
         {
            return false;
         }
         else if ((sslPolicyErrors & SslPolicyErrors.RemoteCertificateNameMismatch) ==
            SslPolicyErrors.RemoteCertificateNameMismatch)
         {
            Zone z;
            z = Zone.CreateFromUrl(((HttpWebRequest)sender).RequestUri.ToString());
            if (z.SecurityZone == System.Security.SecurityZone.Intranet
              || z.SecurityZone == System.Security.SecurityZone.MyComputer)
            {
               return true;
            }
            return false;
         }
         return true;
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      public void Login()
      {
         HttpWebResponse response = null;
         try
         {
            logger_.DebugFormat("logging in");

            Config.Login configLogin = Program.Settings.Login.Value;
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(configLogin.Url);

            request.Method = "POST";
            request.KeepAlive = true;
            request.ProtocolVersion = HttpVersion.Version11;
            //req.UserAgent = "Mozilla/5.0 (Windows NT 6.1; rv:10.0.2) Gecko/20100101 Firefox/10.0.2";
            request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
            request.Headers["DNT"] = "1";
            request.Referer = LoginServer;
            request.Proxy = null;
            request.ContentType = "application/x-www-form-urlencoded";
            string cmd = string.Format(configLogin.ActionFormat, UserName, Password);
            byte[] data = encoding_.GetBytes(cmd);
            request.ContentLength = data.Length;
            Stream newStream = request.GetRequestStream();
            newStream.Write(data, 0, data.Length);
            newStream.Close();

            response = request.GetResponse() as HttpWebResponse;
            string[] elements = response.Headers["Set-Cookie"].Split(',', ';');
            sessionId_ = elements.Where(element => element.StartsWith("JSESSIONID")).First().Split('=')[1];
            
            logger_.DebugFormat("logged in as {0}", sessionId_);
            codeName_ = string.Empty;
            codeDate_ = DateTime.MinValue;
         }
         finally
         {
            if (null != response)
            {
               response.Close();
            }
         }
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      public HttpWebRequest BuildRequest(string url, string verb)
      {
         HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
         request.Method = verb;
         request.Host = LoginServer;
         request.KeepAlive = true;
         request.ProtocolVersion = HttpVersion.Version11;
         request.UserAgent = "Mozilla/5.0 (Windows NT 6.1; rv:10.0.2) Gecko/20100101 Firefox/10.0.2";
         request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
         request.Headers["Accept-Language"] = "en-gb,en;q=0.5";
         request.Headers["Accept-Encoding"] = "gzip, deflate";
         request.Headers["DNT"] = "1";
         request.Referer = LoginServer;
         request.Proxy = null;
         request.ContentType = "application/x-www-form-urlencoded";
         request.Headers["Cookie"] = string.Format(Program.Settings.Page.Value.CookieFormat, sessionId_);
         return request;
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      public HtmlDocument Download(HttpWebRequest request, string cmd, EventWaitHandle stopEvent, string debugName)
      {
         HtmlDocument doc = null;
         HttpWebResponse response = null;
         try
         {
            EventWaitHandle completed = new EventWaitHandle(false, EventResetMode.ManualReset);

            // Request the page with encoded command or as-is
            if (null != cmd)
            {
               byte[] data = encoding_.GetBytes(cmd);
               request.ContentLength = data.Length;
               request.BeginGetRequestStream((streamResult) =>
               {
                  Stream newStream = request.EndGetRequestStream(streamResult);
                  newStream.Write(data, 0, data.Length);
                  newStream.Close();

                  request.BeginGetResponse((result) =>
                  {
                     try
                     {
                        response = request.EndGetResponse(result) as HttpWebResponse;
                        completed.Set();
                     }
                     catch (System.Exception ex)
                     {
                        logger_.Exception(ex);
                        stopEvent.Set();
                     }
                  }, null);
               }, null);
            }
            else
            {
               request.BeginGetResponse((result) =>
               {
                  try
                  {
                     response = request.EndGetResponse(result) as HttpWebResponse;
                     completed.Set();
                  }
                  catch (System.Exception ex)
                  {
                     logger_.Exception(ex);
                     stopEvent.Set();
                  }
               }, null);
            }

            // Wait for the response, or the cancel request
            int eventIndex = EventWaitHandle.WaitAny(new EventWaitHandle[] { stopEvent, completed });
            if (0 == eventIndex)
            {
               logger_.DebugFormat("cancelling request for '{0}'", debugName);
               if ((null != request) && !completed.WaitOne(0))
               {
                  request.Abort();
               }
               return null;
            }

            doc = new HtmlDocument();
            using (Stream responseStream = response.GetResponseStream())
            {
               Stream docStream = responseStream;
               string encoding = response.Headers["Content-Encoding"];
               if ("gzip" == encoding)
               {
                  docStream = new GZipStream(responseStream, CompressionMode.Decompress);
               }
               else if ("deflate" == encoding)
               {
                  docStream = new DeflateStream(responseStream, CompressionMode.Decompress);
               }

               doc.Load(docStream);
               if (null == doc.DocumentNode)
               {
                  logger_.ErrorFormat("failed to parse page image for '{0}', cmd='{1}'",
                     request.RequestUri, cmd ?? string.Empty);
                  //doc.ParseErrors.ToList().ForEach(parseError =>
                  //   {
                  //      logger_.ErrorFormat("   ({0}, {1}): {2}", parseError.Line, parseError.LinePosition, parseError.Reason);
                  //      logger_.ErrorFormat("      {0}", parseError.SourceText);
                  //   });
                  return null;
               }
            }
         }
         catch (System.Exception ex)
         {
            logger_.Exception(ex);
            doc = null;
         }
         finally
         {
            if (null != response)
            {
               response.Close();
            }
         }
         return doc;
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      public HtmlDocument DownloadIntraday(EventWaitHandle stopEvent, Code code,
         DateTime codeDate)
      {
         HtmlDocument doc = null;
         try
         {
            Config.Page configPage = Program.Settings.Page.Value;
            string url = string.Format(configPage.IntradayFormat, code.Name, codeDate);
            HttpWebRequest request = BuildRequest(url, "GET");
            doc = Download(request, null, stopEvent, code.DebugName(codeDate));
         }
         catch (System.Exception ex)
         {
            logger_.Exception(ex);
            doc = null;
         }
         return doc;
      }

      ////////////////////////////////////////////////////////////////////////
      /// <summary>
      /// 
      /// </summary>
      public HtmlDocument Download(EventWaitHandle stopEvent, Code code,
         DateTime codeDate, int pageIndex)
      {
         HtmlDocument doc = null;
         try
         {
            Config.Page configPage = Program.Settings.Page.Value;

            // The default is to use GET and request the page directly
            string verb = "GET";
            string url = string.Format(configPage.GetFormat, code.Name, codeDate, pageIndex);
            string cmd = null;

            // If we have changed code or date, we must use POST to update the server's idea of the
            // entries that we want to collect
            if (string.IsNullOrEmpty(codeName_) || (codeName_ != code.Name) || (codeDate_ != codeDate))
            {
               // If this is not also pageIndex 0, we should do an initial extra call to get page zero
               // and then revert to using GET with the specific pageIndex. Otherwise, we will not be
               // able to get back into alignment
               if (0 < pageIndex)
               {
                  // Call ourselves recursively to post a request for page zero.
                  Download(stopEvent, code, codeDate, 0);

                  // And then carry on with the explicit request for the page that we really want
               }
               else
               {
                  // Swapping code, so use POST to align the cursor
                  
                  logger_.DebugFormat("swapping from name='{0}', date={1:dd/MM/yyyy} to name='{2}', date={3:dd/MM/yyyy}",
                     codeName_, codeDate_, code.Name, codeDate);
                  
                  url = configPage.Url;
                  cmd = string.Format(configPage.PostFormat, code.Name, codeDate, pageIndex);
                  verb = "POST";
                  codeName_ = code.Name;
                  codeDate_ = codeDate;
               }
            }

            HttpWebRequest request = BuildRequest(url, verb);
            doc = Download(request, cmd, stopEvent, code.DebugName(codeDate, pageIndex));

            // Does it look like the page that we are expecting?
            HtmlNode rootNode = (null != doc) ? doc.DocumentNode : null;
            HtmlNode footerNode = (null != rootNode) ? rootNode.SelectSingleNode("//div[@id=\"footer-container\"]") : null;
            if (null == footerNode)
            {
               logger_.DebugFormat("image for '{0}', cmd='{1}' has no footer-container",
                  request.RequestUri, cmd ?? string.Empty);
               return null;
            }

            // Strip all unwanted nodes
            rootNode.SelectNodes("//img").ToList().ForEach(node => node.Remove());

            rootNode.SelectNodes("//form").ToList().ForEach(node => node.Remove());

            NodeLevel level = NodeLevel.LookingForHtml;
            stripNodes(rootNode.ChildNodes, ref level);
         }
         catch (System.Exception ex)
         {
            logger_.Exception(ex);
            doc = null;
         }
         return doc;
      }

      ////////////////////////////////////////////////////////////////////////
      //
      private void stripNodes(HtmlNodeCollection nodes, ref NodeLevel level)
      {
         for (int index = 0; index < nodes.Count; /**/ )
         {
            HtmlNode node = nodes[index];
            switch (level)
            {
               case NodeLevel.LookingForHtml:
                  // The only valid node is <html>
                  if ("html" != node.Name)
                  {
                     nodes.Remove(node);
                  }
                  else
                  {
                     if (node.HasChildNodes)
                     {
                        NodeLevel newLevel = NodeLevel.LookingForBody;
                        stripNodes(node.ChildNodes, ref newLevel);
                        if (NodeLevel.StripTail == newLevel)
                        {
                           // Reached the end
                           level = newLevel;
                        }
                     }
                     ++index;
                  }
                  break;

               case NodeLevel.LookingForBody:
                  // The only valid node is <body>
                  if ("body" != node.Name)
                  {
                     nodes.Remove(node);
                  }
                  else
                  {
                     if (node.HasChildNodes)
                     {
                        NodeLevel newLevel = NodeLevel.InBody;
                        stripNodes(node.ChildNodes, ref newLevel);
                        if (NodeLevel.StripTail == newLevel)
                        {
                           // Reached the end
                           level = newLevel;
                        }
                     }
                     ++index;
                  }
                  break;

               case NodeLevel.InBody:
                  if (("#text" == node.Name) && node.InnerText.Contains("Disclaimer"))
                  {
                     nodes.Remove(node);
                     level = NodeLevel.StripTail;
                  }
                  else if (("script" == node.Name)   // discard all scripts
                     || ("noscript" == node.Name)
                     || ("form" == node.Name)
                     || ("input" == node.Name)
                     || ("select" == node.Name)
                     || ("/form" == node.Name)
                     || ("iframe" == node.Name)
                     || ("#comment" == node.Name)
                     || (("#text" == node.Name) && string.IsNullOrWhiteSpace(node.InnerText))
                     || node.GetAttributeValue("id", "").StartsWith("google_ads")
                     || ("header-container" == node.GetAttributeValue("id", ""))
                     )
                  {
                     nodes.Remove(node);
                  }
                  else
                  {
                     if (node.HasChildNodes)
                     {
                        NodeLevel newLevel = level;
                        stripNodes(node.ChildNodes, ref newLevel);
                        if (NodeLevel.StripTail == newLevel)
                        {
                           // Reached the end
                           level = newLevel;
                        }
                     }

                     // drop this node if it is now an empty div
                     if (!node.HasChildNodes && ("div" == node.Name))
                     {
                        nodes.Remove(node);
                     }
                     else
                     {
                        ++index;
                     }
                  }
                  break;

               case NodeLevel.StripTail:
                  nodes.Remove(node);
                  break;

               default:
                  throw new InvalidOperationException(string.Format("unrecognized NodeLevel enum '{0}'", level));
            }
         }
      }
   }
}
